{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Examine notebook used to visualize results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First we will load the endpoint name, training time, prediction length and seom of the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "endpoint name  DeepAR-forecast-taxidata-2019-12-30-18-32-45-715\n",
      "end training 2019-05-06 00:00:00\n",
      "prediction_length 14\n"
     ]
    }
   ],
   "source": [
    "%store -r\n",
    "print('endpoint name ', endpoint_name)\n",
    "print('end training', end_training)\n",
    "print('prediction_length', prediction_length)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sample data being used:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data sample\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>green</th>\n",
       "      <th>fhvhv</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ts_resampled</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2018-01-01</th>\n",
       "      <td>23292</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2018-01-02</th>\n",
       "      <td>23222</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2018-01-03</th>\n",
       "      <td>26417</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2018-01-04</th>\n",
       "      <td>6519</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2018-01-05</th>\n",
       "      <td>27453</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              green  fhvhv\n",
       "ts_resampled              \n",
       "2018-01-01    23292    0.0\n",
       "2018-01-02    23222    0.0\n",
       "2018-01-03    26417    0.0\n",
       "2018-01-04     6519    0.0\n",
       "2018-01-05    27453    0.0"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('data sample')\n",
    "ABB.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This next cell creates the predictor using the endpoint_name.  Ideally we'd have the DeepARPredictor in a seperate .py rather than repeated in the two notebooks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sagemaker\n",
    "from sagemaker import get_execution_role\n",
    "from sagemaker.tuner import HyperparameterTuner\n",
    "import numpy as np\n",
    "import json\n",
    "import pandas as pd\n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "class DeepARPredictor(sagemaker.predictor.RealTimePredictor):\n",
    "    \n",
    "    def __init__(self, *args, **kwargs):\n",
    "        super().__init__(*args, content_type=sagemaker.content_types.CONTENT_TYPE_JSON, **kwargs)\n",
    "        \n",
    "    def predict(self, ts, cat=None, dynamic_feat=None, \n",
    "                num_samples=100, return_samples=False, quantiles=[\"0.1\", \"0.5\", \"0.9\"]):\n",
    "        \"\"\"Requests the prediction of for the time series listed in `ts`, each with the (optional)\n",
    "        corresponding category listed in `cat`.\n",
    "        \n",
    "        ts -- `pandas.Series` object, the time series to predict\n",
    "        cat -- integer, the group associated to the time series (default: None)\n",
    "        num_samples -- integer, number of samples to compute at prediction time (default: 100)\n",
    "        return_samples -- boolean indicating whether to include samples in the response (default: False)\n",
    "        quantiles -- list of strings specifying the quantiles to compute (default: [\"0.1\", \"0.5\", \"0.9\"])\n",
    "        \n",
    "        Return value: list of `pandas.DataFrame` objects, each containing the predictions\n",
    "        \"\"\"\n",
    "        prediction_time = ts.index[-1] + 1\n",
    "        quantiles = [str(q) for q in quantiles]\n",
    "        req = self.__encode_request(ts, cat, dynamic_feat, num_samples, return_samples, quantiles)\n",
    "        res = super(DeepARPredictor, self).predict(req)\n",
    "        return self.__decode_response(res, ts.index.freq, prediction_time, return_samples)\n",
    "    \n",
    "    def __encode_request(self, ts, cat, dynamic_feat, num_samples, return_samples, quantiles):\n",
    "        instance = series_to_dict(ts, cat if cat is not None else None, dynamic_feat if dynamic_feat else None)\n",
    "\n",
    "        configuration = {\n",
    "            \"num_samples\": num_samples,\n",
    "            \"output_types\": [\"quantiles\", \"samples\"] if return_samples else [\"quantiles\"],\n",
    "            \"quantiles\": quantiles\n",
    "        }\n",
    "        \n",
    "        http_request_data = {\n",
    "            \"instances\": [instance],\n",
    "            \"configuration\": configuration\n",
    "        }\n",
    "        \n",
    "        return json.dumps(http_request_data).encode('utf-8')\n",
    "    \n",
    "    def __decode_response(self, response, freq, prediction_time, return_samples):\n",
    "        # we only sent one time series so we only receive one in return\n",
    "        # however, if possible one will pass multiple time series as predictions will then be faster\n",
    "        predictions = json.loads(response.decode('utf-8'))['predictions'][0]\n",
    "        prediction_length = len(next(iter(predictions['quantiles'].values())))\n",
    "        prediction_index = pd.DatetimeIndex(start=prediction_time, freq=freq, periods=prediction_length)        \n",
    "        if return_samples:\n",
    "            dict_of_samples = {'sample_' + str(i): s for i, s in enumerate(predictions['samples'])}\n",
    "        else:\n",
    "            dict_of_samples = {}\n",
    "        return pd.DataFrame(data={**predictions['quantiles'], **dict_of_samples}, index=prediction_index)\n",
    "\n",
    "    def set_frequency(self, freq):\n",
    "        self.freq = freq\n",
    "        \n",
    "def encode_target(ts):\n",
    "    return [x if np.isfinite(x) else \"NaN\" for x in ts]        \n",
    "\n",
    "def series_to_dict(ts, cat=None, dynamic_feat=None):\n",
    "    \"\"\"Given a pandas.Series object, returns a dictionary encoding the time series.\n",
    "\n",
    "    ts -- a pands.Series object with the target time series\n",
    "    cat -- an integer indicating the time series category\n",
    "\n",
    "    Return value: a dictionary\n",
    "    \"\"\"\n",
    "    obj = {\"start\": str(ts.index[0]), \"target\": encode_target(ts)}\n",
    "    if cat is not None:\n",
    "        obj[\"cat\"] = cat\n",
    "    if dynamic_feat is not None:\n",
    "        obj[\"dynamic_feat\"] = dynamic_feat        \n",
    "    return obj\n",
    "\n",
    "predictor = DeepARPredictor(endpoint_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def plot(\n",
    "    predictor, \n",
    "    target_ts, \n",
    "    cat=None, \n",
    "    dynamic_feat=None, \n",
    "    forecast_date=end_training, \n",
    "    show_samples=False, \n",
    "    plot_history=7 * 12,\n",
    "    confidence=80,\n",
    "    num_samples=100,\n",
    "    draw_color='blue'\n",
    "):\n",
    "    print(\"Calling endpoint to generate {} predictions starting from {} ...\".format(target_ts.name, str(forecast_date)))\n",
    "    assert(confidence > 50 and confidence < 100)\n",
    "    low_quantile = 0.5 - confidence * 0.005\n",
    "    up_quantile = confidence * 0.005 + 0.5\n",
    "        \n",
    "    # we first construct the argument to call our model\n",
    "    args = {\n",
    "        \"ts\": target_ts[:forecast_date],\n",
    "        \"return_samples\": show_samples,\n",
    "        \"quantiles\": [low_quantile, 0.5, up_quantile],\n",
    "        \"num_samples\": num_samples\n",
    "    }\n",
    "\n",
    "\n",
    "    if dynamic_feat is not None:\n",
    "        args[\"dynamic_feat\"] = dynamic_feat\n",
    "        fig = plt.figure(figsize=(20, 6))\n",
    "        ax = plt.subplot(2, 1, 1)\n",
    "    else:\n",
    "        fig = plt.figure(figsize=(20, 3))\n",
    "        ax = plt.subplot(1,1,1)\n",
    "    \n",
    "    if cat is not None:\n",
    "        args[\"cat\"] = cat\n",
    "        ax.text(0.9, 0.9, 'cat = {}'.format(cat), transform=ax.transAxes)\n",
    "\n",
    "    # call the end point to get the prediction\n",
    "    prediction = predictor.predict(**args)\n",
    "\n",
    "    # plot the samples\n",
    "    mccolor = draw_color\n",
    "    if show_samples: \n",
    "        for key in prediction.keys():\n",
    "            if \"sample\" in key:\n",
    "                prediction[key].asfreq('D').plot(color='lightskyblue', alpha=0.2, label='_nolegend_')\n",
    "                \n",
    "    # the date didn't have a frequency in it, so setting it here.\n",
    "    new_date = pd.Timestamp(forecast_date, freq='d')\n",
    "    target_section = target_ts[new_date-plot_history:new_date+prediction_length]\n",
    "    target_section.asfreq('D').plot(color=\"black\", label='target')\n",
    "    plt.title(target_ts.name.upper(), color='darkred')\n",
    "    \n",
    "    # plot the confidence interval and the median predicted\n",
    "    ax.fill_between(\n",
    "        prediction[str(low_quantile)].index, \n",
    "        prediction[str(low_quantile)].values, \n",
    "        prediction[str(up_quantile)].values, \n",
    "        color=mccolor, alpha=0.3, label='{}% confidence interval'.format(confidence)\n",
    "    )\n",
    "    prediction[\"0.5\"].plot(color=mccolor, label='P50')\n",
    "    ax.legend(loc=2)    \n",
    "    \n",
    "    # fix the scale as the samples may change it\n",
    "    ax.set_ylim(target_section.min() * 0.5, target_section.max() * 1.5)\n",
    "    \n",
    "    if dynamic_feat is not None:\n",
    "        for i, f in enumerate(dynamic_feat, start=1):\n",
    "            ax = plt.subplot(len(dynamic_feat) * 2, 1, len(dynamic_feat) + i, sharex=ax)\n",
    "            feat_ts = pd.Series(\n",
    "                index=pd.DatetimeIndex(start=target_ts.index[0], freq=target_ts.index.freq, periods=len(f)),\n",
    "                data=f\n",
    "            )\n",
    "            feat_ts[forecast_date-plot_history:forecast_date+prediction_length].plot(ax=ax, color='g')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's interact w/ the samples and forecast values now."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0baad34f9a854e93bf4dd983b20d2358",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(RadioButtons(description='Type', index=1, options=('full_fhv', 'yellow', 'green'), value…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "from ipywidgets import interact, interactive, fixed, interact_manual\n",
    "import ipywidgets as widgets\n",
    "from ipywidgets import IntSlider, FloatSlider, Checkbox, RadioButtons\n",
    "import datetime\n",
    "\n",
    "style = {'description_width': 'initial'}\n",
    "\n",
    "@interact_manual(\n",
    "    series_type=RadioButtons(options=['full_fhv', 'yellow', 'green'], value='yellow', description='Type'),\n",
    "    forecast_day=IntSlider(min=0, max=100, value=21, style=style),\n",
    "    confidence=IntSlider(min=60, max=95, value=80, step=5, style=style),\n",
    "    history_weeks_plot=IntSlider(min=1, max=20, value=4, style=style),\n",
    "    num_samples=IntSlider(min=100, max=1000, value=100, step=500, style=style),\n",
    "    show_samples=Checkbox(value=True),\n",
    "    continuous_update=False\n",
    ")\n",
    "\n",
    "def plot_interact(series_type, forecast_day, confidence, history_weeks_plot, show_samples, num_samples):   \n",
    "    plot(\n",
    "        predictor,\n",
    "        target_ts=ABB[series_type].asfreq(freq='d', fill_value=0),\n",
    "        forecast_date=end_training + datetime.timedelta(days=forecast_day),\n",
    "        show_samples=show_samples,\n",
    "        plot_history=history_weeks_plot * prediction_length,\n",
    "        confidence=confidence,\n",
    "        num_samples=num_samples\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing and Understanding the results\n",
    "You can test the results and see different results. Here are some examples below:\n",
    "\n",
    "![sample_result1](images/sample_result1.png \"\")\n",
    "![sample_result2](images/sample_result2.png \"\")\n",
    "![sample_result3](images/sample_result3.png \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
